{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6.3 PyTorchでDueling Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 動画の描画関数の宣言\n",
    "# 参考URL http://nbviewer.jupyter.org/github/patrickmineault\n",
    "# /xcorr-notebooks/blob/master/Render%20OpenAI%20gym%20as%20GIF.ipynb\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "from matplotlib import animation\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    \"\"\"\n",
    "    Displays a list of frames as a gif, with controls\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(frames[0].shape[1] / 72.0, frames[0].shape[0] / 72.0),\n",
    "               dpi=72)\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "\n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames=len(frames),\n",
    "                                   interval=50)\n",
    "\n",
    "    anim.save('movie_cartpole_dueling_network.mp4')  # 動画のファイル名と保存です\n",
    "    display(display_animation(anim, default_mode='loop'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# namedtupleを生成\n",
    "from collections import namedtuple\n",
    "\n",
    "Transition = namedtuple(\n",
    "    'Transition', ('state', 'action', 'next_state', 'reward'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定数の設定\n",
    "ENV = 'CartPole-v0'  # 使用する課題名\n",
    "GAMMA = 0.99  # 時間割引率\n",
    "MAX_STEPS = 200  # 1試行のstep数\n",
    "NUM_EPISODES = 500  # 最大試行回数\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 経験を保存するメモリクラスを定義します\n",
    "\n",
    "\n",
    "class ReplayMemory:\n",
    "\n",
    "    def __init__(self, CAPACITY):\n",
    "        self.capacity = CAPACITY  # メモリの最大長さ\n",
    "        self.memory = []  # 経験を保存する変数\n",
    "        self.index = 0  # 保存するindexを示す変数\n",
    "\n",
    "    def push(self, state, action, state_next, reward):\n",
    "        '''transition = (state, action, state_next, reward)をメモリに保存する'''\n",
    "\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)  # メモリが満タンでないときは足す\n",
    "\n",
    "        # namedtupleのTransitionを使用し、値とフィールド名をペアにして保存します\n",
    "        self.memory[self.index] = Transition(state, action, state_next, reward)\n",
    "\n",
    "        self.index = (self.index + 1) % self.capacity  # 保存するindexを1つずらす\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        '''batch_size分だけ、ランダムに保存内容を取り出す'''\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        '''関数lenに対して、現在の変数memoryの長さを返す'''\n",
    "        return len(self.memory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dueling Network型のディープ・ニューラルネットワークの構築\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self, n_in, n_mid, n_out):\n",
    "        super(Net, self).__init__()\n",
    "        self.fc1 = nn.Linear(n_in, n_mid)\n",
    "        self.fc2 = nn.Linear(n_mid, n_mid)\n",
    "        # Dueling Network\n",
    "        self.fc3_adv = nn.Linear(n_mid, n_out)  # Advantage側\n",
    "        self.fc3_v = nn.Linear(n_mid, 1)  # 価値V側\n",
    "\n",
    "    def forward(self, x):\n",
    "        h1 = F.relu(self.fc1(x))\n",
    "        h2 = F.relu(self.fc2(h1))\n",
    "\n",
    "        adv = self.fc3_adv(h2)  # この出力はReLUしない\n",
    "        val = self.fc3_v(h2).expand(-1, adv.size(1))  # この出力はReLUしない\n",
    "        # valはadvと足し算するために、サイズを[minibatchx1]から[minibatchx2]にexpandする\n",
    "        # adv.size(1)は出力する行動の種類数の2\n",
    "\n",
    "        output = val + adv - adv.mean(1, keepdim=True).expand(-1, adv.size(1))\n",
    "        # val+advからadvの平均値を引き算する\n",
    "        # adv.mean(1, keepdim=True) で列方向(行動の種類方向)に平均し、サイズが[minibatch×1]\n",
    "        # expandで展開して、サイズ[minibatchx2]\n",
    "\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# エージェントが持つ脳となるクラスです、DDQNを実行します\n",
    "\n",
    "import random\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "CAPACITY = 10000\n",
    "\n",
    "\n",
    "class Brain:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        self.num_actions = num_actions  # CartPoleの行動（右に左に押す）の2を取得\n",
    "\n",
    "        # 経験を記憶するメモリオブジェクトを生成\n",
    "        self.memory = ReplayMemory(CAPACITY)\n",
    "\n",
    "        # ニューラルネットワークを構築\n",
    "        n_in, n_mid, n_out = num_states, 32, num_actions\n",
    "        self.main_q_network = Net(n_in, n_mid, n_out)  # Netクラスを使用\n",
    "        self.target_q_network = Net(n_in, n_mid, n_out)  # Netクラスを使用\n",
    "        print(self.main_q_network)  # ネットワークの形を出力\n",
    "\n",
    "        # 最適化手法の設定\n",
    "        self.optimizer = optim.Adam(\n",
    "            self.main_q_network.parameters(), lr=0.0001)\n",
    "\n",
    "    def replay(self):\n",
    "        '''Experience Replayでネットワークの結合パラメータを学習'''\n",
    "\n",
    "        # 1. メモリサイズの確認\n",
    "        if len(self.memory) < BATCH_SIZE:\n",
    "            return\n",
    "\n",
    "        # 2. ミニバッチの作成\n",
    "        self.batch, self.state_batch, self.action_batch, self.reward_batch, self.non_final_next_states = self.make_minibatch()\n",
    "\n",
    "        # 3. 教師信号となるQ(s_t, a_t)値を求める\n",
    "        self.expected_state_action_values = self.get_expected_state_action_values()\n",
    "\n",
    "        # 4. 結合パラメータの更新\n",
    "        self.update_main_q_network()\n",
    "\n",
    "    def decide_action(self, state, episode):\n",
    "        '''現在の状態に応じて、行動を決定する'''\n",
    "        # ε-greedy法で徐々に最適行動のみを採用する\n",
    "        epsilon = 0.5 * (1 / (episode + 1))\n",
    "\n",
    "        if epsilon <= np.random.uniform(0, 1):\n",
    "            self.main_q_network.eval()  # ネットワークを推論モードに切り替える\n",
    "            with torch.no_grad():\n",
    "                action = self.main_q_network(state).max(1)[1].view(1, 1)\n",
    "            # ネットワークの出力の最大値のindexを取り出します = max(1)[1]\n",
    "            # .view(1,1)は[torch.LongTensor of size 1]　を size 1x1 に変換します\n",
    "\n",
    "        else:\n",
    "            # 0,1の行動をランダムに返す\n",
    "            action = torch.LongTensor(\n",
    "                [[random.randrange(self.num_actions)]])  # 0,1の行動をランダムに返す\n",
    "            # actionは[torch.LongTensor of size 1x1]の形になります\n",
    "\n",
    "        return action\n",
    "\n",
    "    def make_minibatch(self):\n",
    "        '''2. ミニバッチの作成'''\n",
    "\n",
    "        # 2.1 メモリからミニバッチ分のデータを取り出す\n",
    "        transitions = self.memory.sample(BATCH_SIZE)\n",
    "\n",
    "        # 2.2 各変数をミニバッチに対応する形に変形\n",
    "        # transitionsは1stepごとの(state, action, state_next, reward)が、BATCH_SIZE分格納されている\n",
    "        # つまり、(state, action, state_next, reward)×BATCH_SIZE\n",
    "        # これをミニバッチにしたい。つまり\n",
    "        # (state×BATCH_SIZE, action×BATCH_SIZE, state_next×BATCH_SIZE, reward×BATCH_SIZE)にする\n",
    "        batch = Transition(*zip(*transitions))\n",
    "\n",
    "        # 2.3 各変数の要素をミニバッチに対応する形に変形し、ネットワークで扱えるようVariableにする\n",
    "        # 例えばstateの場合、[torch.FloatTensor of size 1x4]がBATCH_SIZE分並んでいるのですが、\n",
    "        # それを torch.FloatTensor of size BATCH_SIZEx4 に変換します\n",
    "        # 状態、行動、報酬、non_finalの状態のミニバッチのVariableを作成\n",
    "        # catはConcatenates（結合）のことです。\n",
    "        state_batch = torch.cat(batch.state)\n",
    "        action_batch = torch.cat(batch.action)\n",
    "        reward_batch = torch.cat(batch.reward)\n",
    "        non_final_next_states = torch.cat([s for s in batch.next_state\n",
    "                                           if s is not None])\n",
    "\n",
    "        return batch, state_batch, action_batch, reward_batch, non_final_next_states\n",
    "\n",
    "    def get_expected_state_action_values(self):\n",
    "        '''3. 教師信号となるQ(s_t, a_t)値を求める'''\n",
    "\n",
    "        # 3.1 ネットワークを推論モードに切り替える\n",
    "        self.main_q_network.eval()\n",
    "        self.target_q_network.eval()\n",
    "\n",
    "        # 3.2 ネットワークが出力したQ(s_t, a_t)を求める\n",
    "        # self.model(state_batch)は、右左の両方のQ値を出力しており\n",
    "        # [torch.FloatTensor of size BATCH_SIZEx2]になっている。\n",
    "        # ここから実行したアクションa_tに対応するQ値を求めるため、action_batchで行った行動a_tが右か左かのindexを求め\n",
    "        # それに対応するQ値をgatherでひっぱり出す。\n",
    "        self.state_action_values = self.main_q_network(\n",
    "            self.state_batch).gather(1, self.action_batch)\n",
    "\n",
    "        # 3.3 max{Q(s_t+1, a)}値を求める。ただし次の状態があるかに注意。\n",
    "\n",
    "        # cartpoleがdoneになっておらず、next_stateがあるかをチェックするインデックスマスクを作成\n",
    "        non_final_mask = torch.ByteTensor(tuple(map(lambda s: s is not None,\n",
    "                                                    self.batch.next_state)))\n",
    "        # まずは全部0にしておく\n",
    "        next_state_values = torch.zeros(BATCH_SIZE)\n",
    "\n",
    "        a_m = torch.zeros(BATCH_SIZE).type(torch.LongTensor)\n",
    "\n",
    "        # 次の状態での最大Q値の行動a_mをMain Q-Networkから求める\n",
    "        # 最後の[1]で行動に対応したindexが返る\n",
    "        a_m[non_final_mask] = self.main_q_network(\n",
    "            self.non_final_next_states).detach().max(1)[1]\n",
    "\n",
    "        # 次の状態があるものだけにフィルターし、size 32を32×1へ\n",
    "        a_m_non_final_next_states = a_m[non_final_mask].view(-1, 1)\n",
    "\n",
    "        # 次の状態があるindexの、行動a_mのQ値をtarget Q-Networkから求める\n",
    "        # detach()で取り出す\n",
    "        # squeeze()でsize[minibatch×1]を[minibatch]に。\n",
    "        next_state_values[non_final_mask] = self.target_q_network(\n",
    "            self.non_final_next_states).gather(1, a_m_non_final_next_states).detach().squeeze()\n",
    "\n",
    "        # 3.4 教師となるQ(s_t, a_t)値を、Q学習の式から求める\n",
    "        expected_state_action_values = self.reward_batch + GAMMA * next_state_values\n",
    "\n",
    "        return expected_state_action_values\n",
    "\n",
    "    def update_main_q_network(self):\n",
    "        '''4. 結合パラメータの更新'''\n",
    "\n",
    "        # 4.1 ネットワークを訓練モードに切り替える\n",
    "        self.main_q_network.train()\n",
    "\n",
    "        # 4.2 損失関数を計算する（smooth_l1_lossはHuberloss）\n",
    "        # expected_state_action_valuesは\n",
    "        # sizeが[minbatch]になっているので、unsqueezeで[minibatch x 1]へ\n",
    "        loss = F.smooth_l1_loss(self.state_action_values,\n",
    "                                self.expected_state_action_values.unsqueeze(1))\n",
    "\n",
    "        # 4.3 結合パラメータを更新する\n",
    "        self.optimizer.zero_grad()  # 勾配をリセット\n",
    "        loss.backward()  # バックプロパゲーションを計算\n",
    "        self.optimizer.step()  # 結合パラメータを更新\n",
    "\n",
    "    def update_target_q_network(self):  # DDQNで追加\n",
    "        '''Target Q-NetworkをMainと同じにする'''\n",
    "        self.target_q_network.load_state_dict(self.main_q_network.state_dict())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleで動くエージェントクラスです、棒付き台車そのものになります\n",
    "\n",
    "\n",
    "class Agent:\n",
    "    def __init__(self, num_states, num_actions):\n",
    "        '''課題の状態と行動の数を設定する'''\n",
    "        self.brain = Brain(num_states, num_actions)  # エージェントが行動を決定するための頭脳を生成\n",
    "\n",
    "    def update_q_function(self):\n",
    "        '''Q関数を更新する'''\n",
    "        self.brain.replay()\n",
    "\n",
    "    def get_action(self, state, episode):\n",
    "        '''行動を決定する'''\n",
    "        action = self.brain.decide_action(state, episode)\n",
    "        return action\n",
    "\n",
    "    def memorize(self, state, action, state_next, reward):\n",
    "        '''memoryオブジェクトに、state, action, state_next, rewardの内容を保存する'''\n",
    "        self.brain.memory.push(state, action, state_next, reward)\n",
    "\n",
    "    def update_target_q_function(self):\n",
    "        '''Target Q-NetworkをMain Q-Networkと同じに更新'''\n",
    "        self.brain.update_target_q_network()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CartPoleを実行する環境のクラスです\n",
    "\n",
    "\n",
    "class Environment:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.env = gym.make(ENV)  # 実行する課題を設定\n",
    "        num_states = self.env.observation_space.shape[0]  # 課題の状態と行動の数を設定\n",
    "        num_actions = self.env.action_space.n  # CartPoleの行動（右に左に押す）の2を取得\n",
    "        # 環境内で行動するAgentを生成\n",
    "        self.agent = Agent(num_states, num_actions)\n",
    "\n",
    "    def run(self):\n",
    "        '''実行'''\n",
    "        episode_10_list = np.zeros(10)  # 10試行分の立ち続けたstep数を格納し、平均ステップ数を出力に利用\n",
    "        complete_episodes = 0  # 195step以上連続で立ち続けた試行数\n",
    "        episode_final = False  # 最後の試行フラグ\n",
    "        frames = []  # 最後の試行を動画にするために画像を格納する変数\n",
    "\n",
    "        for episode in range(NUM_EPISODES):  # 試行数分繰り返す\n",
    "            observation = self.env.reset()  # 環境の初期化\n",
    "\n",
    "            state = observation  # 観測をそのまま状態sとして使用\n",
    "            state = torch.from_numpy(state).type(\n",
    "                torch.FloatTensor)  # numpy変数をPyTorchのテンソルに変換\n",
    "            state = torch.unsqueeze(state, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "            for step in range(MAX_STEPS):  # 1エピソードのループ\n",
    "                \n",
    "                # 動画描画をコメントアウトしています\n",
    "                #if episode_final is True:  # 最終試行ではframesに各時刻の画像を追加していく\n",
    "                    # frames.append(self.env.render(mode='rgb_array'))\n",
    "                    \n",
    "                action = self.agent.get_action(state, episode)  # 行動を求める\n",
    "\n",
    "                # 行動a_tの実行により、s_{t+1}とdoneフラグを求める\n",
    "                # actionから.item()を指定して、中身を取り出す\n",
    "                observation_next, _, done, _ = self.env.step(\n",
    "                    action.item())  # rewardとinfoは使わないので_にする\n",
    "\n",
    "                # 報酬を与える。さらにepisodeの終了評価と、state_nextを設定する\n",
    "                if done:  # ステップ数が200経過するか、一定角度以上傾くとdoneはtrueになる\n",
    "                    state_next = None  # 次の状態はないので、Noneを格納\n",
    "\n",
    "                    # 直近10episodeの立てたstep数リストに追加\n",
    "                    episode_10_list = np.hstack(\n",
    "                        (episode_10_list[1:], step + 1))\n",
    "\n",
    "                    if step < 195:\n",
    "                        reward = torch.FloatTensor(\n",
    "                            [-1.0])  # 途中でこけたら罰則として報酬-1を与える\n",
    "                        complete_episodes = 0  # 連続成功記録をリセット\n",
    "                    else:\n",
    "                        reward = torch.FloatTensor([1.0])  # 立ったまま終了時は報酬1を与える\n",
    "                        complete_episodes = complete_episodes + 1  # 連続記録を更新\n",
    "                else:\n",
    "                    reward = torch.FloatTensor([0.0])  # 普段は報酬0\n",
    "                    state_next = observation_next  # 観測をそのまま状態とする\n",
    "                    state_next = torch.from_numpy(state_next).type(\n",
    "                        torch.FloatTensor)  # numpy変数をPyTorchのテンソルに変換\n",
    "                    state_next = torch.unsqueeze(state_next, 0)  # size 4をsize 1x4に変換\n",
    "\n",
    "                # メモリに経験を追加\n",
    "                self.agent.memorize(state, action, state_next, reward)\n",
    "\n",
    "                # Experience ReplayでQ関数を更新する\n",
    "                self.agent.update_q_function()\n",
    "\n",
    "                # 観測の更新\n",
    "                state = state_next\n",
    "\n",
    "                # 終了時の処理\n",
    "                if done:\n",
    "                    print('%d Episode: Finished after %d steps：10試行の平均step数 = %.1lf' % (\n",
    "                        episode, step + 1, episode_10_list.mean()))\n",
    "                    \n",
    "                    # DDQNで追加、2試行に1度、Target Q-NetworkをMainと同じにコピーする\n",
    "                    if(episode % 2 == 0):\n",
    "                        self.agent.update_target_q_function()\n",
    "                    break\n",
    "                    \n",
    "                    \n",
    "            if episode_final is True:\n",
    "                # 動画描画をコメントアウトしています\n",
    "                # 動画を保存と描画\n",
    "                #display_frames_as_gif(frames)\n",
    "                break\n",
    "\n",
    "            # 10連続で200step経ち続けたら成功\n",
    "            if complete_episodes >= 10:\n",
    "                print('10回連続成功')\n",
    "                episode_final = True  # 次の試行を描画を行う最終試行とする\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Net(\n",
      "  (fc1): Linear(in_features=4, out_features=32, bias=True)\n",
      "  (fc2): Linear(in_features=32, out_features=32, bias=True)\n",
      "  (fc3_adv): Linear(in_features=32, out_features=2, bias=True)\n",
      "  (fc3_v): Linear(in_features=32, out_features=1, bias=True)\n",
      ")\n",
      "0 Episode: Finished after 9 steps：10試行の平均step数 = 0.9\n",
      "1 Episode: Finished after 10 steps：10試行の平均step数 = 1.9\n",
      "2 Episode: Finished after 9 steps：10試行の平均step数 = 2.8\n",
      "3 Episode: Finished after 10 steps：10試行の平均step数 = 3.8\n",
      "4 Episode: Finished after 10 steps：10試行の平均step数 = 4.8\n",
      "5 Episode: Finished after 9 steps：10試行の平均step数 = 5.7\n",
      "6 Episode: Finished after 9 steps：10試行の平均step数 = 6.6\n",
      "7 Episode: Finished after 10 steps：10試行の平均step数 = 7.6\n",
      "8 Episode: Finished after 12 steps：10試行の平均step数 = 8.8\n",
      "9 Episode: Finished after 14 steps：10試行の平均step数 = 10.2\n",
      "10 Episode: Finished after 14 steps：10試行の平均step数 = 10.7\n",
      "11 Episode: Finished after 15 steps：10試行の平均step数 = 11.2\n",
      "12 Episode: Finished after 14 steps：10試行の平均step数 = 11.7\n",
      "13 Episode: Finished after 16 steps：10試行の平均step数 = 12.3\n",
      "14 Episode: Finished after 13 steps：10試行の平均step数 = 12.6\n",
      "15 Episode: Finished after 11 steps：10試行の平均step数 = 12.8\n",
      "16 Episode: Finished after 15 steps：10試行の平均step数 = 13.4\n",
      "17 Episode: Finished after 13 steps：10試行の平均step数 = 13.7\n",
      "18 Episode: Finished after 15 steps：10試行の平均step数 = 14.0\n",
      "19 Episode: Finished after 11 steps：10試行の平均step数 = 13.7\n",
      "20 Episode: Finished after 13 steps：10試行の平均step数 = 13.6\n",
      "21 Episode: Finished after 11 steps：10試行の平均step数 = 13.2\n",
      "22 Episode: Finished after 15 steps：10試行の平均step数 = 13.3\n",
      "23 Episode: Finished after 11 steps：10試行の平均step数 = 12.8\n",
      "24 Episode: Finished after 13 steps：10試行の平均step数 = 12.8\n",
      "25 Episode: Finished after 14 steps：10試行の平均step数 = 13.1\n",
      "26 Episode: Finished after 10 steps：10試行の平均step数 = 12.6\n",
      "27 Episode: Finished after 13 steps：10試行の平均step数 = 12.6\n",
      "28 Episode: Finished after 15 steps：10試行の平均step数 = 12.6\n",
      "29 Episode: Finished after 15 steps：10試行の平均step数 = 13.0\n",
      "30 Episode: Finished after 13 steps：10試行の平均step数 = 13.0\n",
      "31 Episode: Finished after 13 steps：10試行の平均step数 = 13.2\n",
      "32 Episode: Finished after 17 steps：10試行の平均step数 = 13.4\n",
      "33 Episode: Finished after 19 steps：10試行の平均step数 = 14.2\n",
      "34 Episode: Finished after 15 steps：10試行の平均step数 = 14.4\n",
      "35 Episode: Finished after 17 steps：10試行の平均step数 = 14.7\n",
      "36 Episode: Finished after 13 steps：10試行の平均step数 = 15.0\n",
      "37 Episode: Finished after 24 steps：10試行の平均step数 = 16.1\n",
      "38 Episode: Finished after 17 steps：10試行の平均step数 = 16.3\n",
      "39 Episode: Finished after 16 steps：10試行の平均step数 = 16.4\n",
      "40 Episode: Finished after 12 steps：10試行の平均step数 = 16.3\n",
      "41 Episode: Finished after 14 steps：10試行の平均step数 = 16.4\n",
      "42 Episode: Finished after 14 steps：10試行の平均step数 = 16.1\n",
      "43 Episode: Finished after 15 steps：10試行の平均step数 = 15.7\n",
      "44 Episode: Finished after 10 steps：10試行の平均step数 = 15.2\n",
      "45 Episode: Finished after 13 steps：10試行の平均step数 = 14.8\n",
      "46 Episode: Finished after 14 steps：10試行の平均step数 = 14.9\n",
      "47 Episode: Finished after 14 steps：10試行の平均step数 = 13.9\n",
      "48 Episode: Finished after 10 steps：10試行の平均step数 = 13.2\n",
      "49 Episode: Finished after 11 steps：10試行の平均step数 = 12.7\n",
      "50 Episode: Finished after 10 steps：10試行の平均step数 = 12.5\n",
      "51 Episode: Finished after 17 steps：10試行の平均step数 = 12.8\n",
      "52 Episode: Finished after 17 steps：10試行の平均step数 = 13.1\n",
      "53 Episode: Finished after 11 steps：10試行の平均step数 = 12.7\n",
      "54 Episode: Finished after 24 steps：10試行の平均step数 = 14.1\n",
      "55 Episode: Finished after 27 steps：10試行の平均step数 = 15.5\n",
      "56 Episode: Finished after 31 steps：10試行の平均step数 = 17.2\n",
      "57 Episode: Finished after 28 steps：10試行の平均step数 = 18.6\n",
      "58 Episode: Finished after 30 steps：10試行の平均step数 = 20.6\n",
      "59 Episode: Finished after 22 steps：10試行の平均step数 = 21.7\n",
      "60 Episode: Finished after 36 steps：10試行の平均step数 = 24.3\n",
      "61 Episode: Finished after 60 steps：10試行の平均step数 = 28.6\n",
      "62 Episode: Finished after 44 steps：10試行の平均step数 = 31.3\n",
      "63 Episode: Finished after 26 steps：10試行の平均step数 = 32.8\n",
      "64 Episode: Finished after 87 steps：10試行の平均step数 = 39.1\n",
      "65 Episode: Finished after 51 steps：10試行の平均step数 = 41.5\n",
      "66 Episode: Finished after 23 steps：10試行の平均step数 = 40.7\n",
      "67 Episode: Finished after 38 steps：10試行の平均step数 = 41.7\n",
      "68 Episode: Finished after 34 steps：10試行の平均step数 = 42.1\n",
      "69 Episode: Finished after 45 steps：10試行の平均step数 = 44.4\n",
      "70 Episode: Finished after 33 steps：10試行の平均step数 = 44.1\n",
      "71 Episode: Finished after 31 steps：10試行の平均step数 = 41.2\n",
      "72 Episode: Finished after 33 steps：10試行の平均step数 = 40.1\n",
      "73 Episode: Finished after 33 steps：10試行の平均step数 = 40.8\n",
      "74 Episode: Finished after 37 steps：10試行の平均step数 = 35.8\n",
      "75 Episode: Finished after 61 steps：10試行の平均step数 = 36.8\n",
      "76 Episode: Finished after 35 steps：10試行の平均step数 = 38.0\n",
      "77 Episode: Finished after 74 steps：10試行の平均step数 = 41.6\n",
      "78 Episode: Finished after 35 steps：10試行の平均step数 = 41.7\n",
      "79 Episode: Finished after 36 steps：10試行の平均step数 = 40.8\n",
      "80 Episode: Finished after 42 steps：10試行の平均step数 = 41.7\n",
      "81 Episode: Finished after 27 steps：10試行の平均step数 = 41.3\n",
      "82 Episode: Finished after 28 steps：10試行の平均step数 = 40.8\n",
      "83 Episode: Finished after 53 steps：10試行の平均step数 = 42.8\n",
      "84 Episode: Finished after 29 steps：10試行の平均step数 = 42.0\n",
      "85 Episode: Finished after 29 steps：10試行の平均step数 = 38.8\n",
      "86 Episode: Finished after 27 steps：10試行の平均step数 = 38.0\n",
      "87 Episode: Finished after 41 steps：10試行の平均step数 = 34.7\n",
      "88 Episode: Finished after 32 steps：10試行の平均step数 = 34.4\n",
      "89 Episode: Finished after 44 steps：10試行の平均step数 = 35.2\n",
      "90 Episode: Finished after 128 steps：10試行の平均step数 = 43.8\n",
      "91 Episode: Finished after 28 steps：10試行の平均step数 = 43.9\n",
      "92 Episode: Finished after 38 steps：10試行の平均step数 = 44.9\n",
      "93 Episode: Finished after 33 steps：10試行の平均step数 = 42.9\n",
      "94 Episode: Finished after 38 steps：10試行の平均step数 = 43.8\n",
      "95 Episode: Finished after 36 steps：10試行の平均step数 = 44.5\n",
      "96 Episode: Finished after 76 steps：10試行の平均step数 = 49.4\n",
      "97 Episode: Finished after 37 steps：10試行の平均step数 = 49.0\n",
      "98 Episode: Finished after 30 steps：10試行の平均step数 = 48.8\n",
      "99 Episode: Finished after 28 steps：10試行の平均step数 = 47.2\n",
      "100 Episode: Finished after 40 steps：10試行の平均step数 = 38.4\n",
      "101 Episode: Finished after 30 steps：10試行の平均step数 = 38.6\n",
      "102 Episode: Finished after 35 steps：10試行の平均step数 = 38.3\n",
      "103 Episode: Finished after 42 steps：10試行の平均step数 = 39.2\n",
      "104 Episode: Finished after 59 steps：10試行の平均step数 = 41.3\n",
      "105 Episode: Finished after 31 steps：10試行の平均step数 = 40.8\n",
      "106 Episode: Finished after 76 steps：10試行の平均step数 = 40.8\n",
      "107 Episode: Finished after 30 steps：10試行の平均step数 = 40.1\n",
      "108 Episode: Finished after 30 steps：10試行の平均step数 = 40.1\n",
      "109 Episode: Finished after 52 steps：10試行の平均step数 = 42.5\n",
      "110 Episode: Finished after 34 steps：10試行の平均step数 = 41.9\n",
      "111 Episode: Finished after 39 steps：10試行の平均step数 = 42.8\n",
      "112 Episode: Finished after 43 steps：10試行の平均step数 = 43.6\n",
      "113 Episode: Finished after 39 steps：10試行の平均step数 = 43.3\n",
      "114 Episode: Finished after 53 steps：10試行の平均step数 = 42.7\n",
      "115 Episode: Finished after 40 steps：10試行の平均step数 = 43.6\n",
      "116 Episode: Finished after 83 steps：10試行の平均step数 = 44.3\n",
      "117 Episode: Finished after 46 steps：10試行の平均step数 = 45.9\n",
      "118 Episode: Finished after 39 steps：10試行の平均step数 = 46.8\n",
      "119 Episode: Finished after 37 steps：10試行の平均step数 = 45.3\n",
      "120 Episode: Finished after 70 steps：10試行の平均step数 = 48.9\n",
      "121 Episode: Finished after 93 steps：10試行の平均step数 = 54.3\n",
      "122 Episode: Finished after 50 steps：10試行の平均step数 = 55.0\n",
      "123 Episode: Finished after 41 steps：10試行の平均step数 = 55.2\n",
      "124 Episode: Finished after 51 steps：10試行の平均step数 = 55.0\n",
      "125 Episode: Finished after 38 steps：10試行の平均step数 = 54.8\n",
      "126 Episode: Finished after 47 steps：10試行の平均step数 = 51.2\n",
      "127 Episode: Finished after 118 steps：10試行の平均step数 = 58.4\n",
      "128 Episode: Finished after 96 steps：10試行の平均step数 = 64.1\n",
      "129 Episode: Finished after 86 steps：10試行の平均step数 = 69.0\n",
      "130 Episode: Finished after 79 steps：10試行の平均step数 = 69.9\n",
      "131 Episode: Finished after 78 steps：10試行の平均step数 = 68.4\n",
      "132 Episode: Finished after 70 steps：10試行の平均step数 = 70.4\n",
      "133 Episode: Finished after 110 steps：10試行の平均step数 = 77.3\n",
      "134 Episode: Finished after 96 steps：10試行の平均step数 = 81.8\n",
      "135 Episode: Finished after 121 steps：10試行の平均step数 = 90.1\n",
      "136 Episode: Finished after 89 steps：10試行の平均step数 = 94.3\n",
      "137 Episode: Finished after 168 steps：10試行の平均step数 = 99.3\n",
      "138 Episode: Finished after 125 steps：10試行の平均step数 = 102.2\n",
      "139 Episode: Finished after 136 steps：10試行の平均step数 = 107.2\n",
      "140 Episode: Finished after 107 steps：10試行の平均step数 = 110.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141 Episode: Finished after 145 steps：10試行の平均step数 = 116.7\n",
      "142 Episode: Finished after 101 steps：10試行の平均step数 = 119.8\n",
      "143 Episode: Finished after 130 steps：10試行の平均step数 = 121.8\n",
      "144 Episode: Finished after 131 steps：10試行の平均step数 = 125.3\n",
      "145 Episode: Finished after 117 steps：10試行の平均step数 = 124.9\n",
      "146 Episode: Finished after 115 steps：10試行の平均step数 = 127.5\n",
      "147 Episode: Finished after 115 steps：10試行の平均step数 = 122.2\n",
      "148 Episode: Finished after 123 steps：10試行の平均step数 = 122.0\n",
      "149 Episode: Finished after 114 steps：10試行の平均step数 = 119.8\n",
      "150 Episode: Finished after 174 steps：10試行の平均step数 = 126.5\n",
      "151 Episode: Finished after 124 steps：10試行の平均step数 = 124.4\n",
      "152 Episode: Finished after 184 steps：10試行の平均step数 = 132.7\n",
      "153 Episode: Finished after 131 steps：10試行の平均step数 = 132.8\n",
      "154 Episode: Finished after 142 steps：10試行の平均step数 = 133.9\n",
      "155 Episode: Finished after 196 steps：10試行の平均step数 = 141.8\n",
      "156 Episode: Finished after 154 steps：10試行の平均step数 = 145.7\n",
      "157 Episode: Finished after 120 steps：10試行の平均step数 = 146.2\n",
      "158 Episode: Finished after 200 steps：10試行の平均step数 = 153.9\n",
      "159 Episode: Finished after 163 steps：10試行の平均step数 = 158.8\n",
      "160 Episode: Finished after 164 steps：10試行の平均step数 = 157.8\n",
      "161 Episode: Finished after 155 steps：10試行の平均step数 = 160.9\n",
      "162 Episode: Finished after 144 steps：10試行の平均step数 = 156.9\n",
      "163 Episode: Finished after 163 steps：10試行の平均step数 = 160.1\n",
      "164 Episode: Finished after 185 steps：10試行の平均step数 = 164.4\n",
      "165 Episode: Finished after 146 steps：10試行の平均step数 = 159.4\n",
      "166 Episode: Finished after 123 steps：10試行の平均step数 = 156.3\n",
      "167 Episode: Finished after 200 steps：10試行の平均step数 = 164.3\n",
      "168 Episode: Finished after 200 steps：10試行の平均step数 = 164.3\n",
      "169 Episode: Finished after 135 steps：10試行の平均step数 = 161.5\n",
      "170 Episode: Finished after 130 steps：10試行の平均step数 = 158.1\n",
      "171 Episode: Finished after 200 steps：10試行の平均step数 = 162.6\n",
      "172 Episode: Finished after 129 steps：10試行の平均step数 = 161.1\n",
      "173 Episode: Finished after 136 steps：10試行の平均step数 = 158.4\n",
      "174 Episode: Finished after 144 steps：10試行の平均step数 = 154.3\n",
      "175 Episode: Finished after 185 steps：10試行の平均step数 = 158.2\n",
      "176 Episode: Finished after 172 steps：10試行の平均step数 = 163.1\n",
      "177 Episode: Finished after 174 steps：10試行の平均step数 = 160.5\n",
      "178 Episode: Finished after 170 steps：10試行の平均step数 = 157.5\n",
      "179 Episode: Finished after 154 steps：10試行の平均step数 = 159.4\n",
      "180 Episode: Finished after 200 steps：10試行の平均step数 = 166.4\n",
      "181 Episode: Finished after 160 steps：10試行の平均step数 = 162.4\n",
      "182 Episode: Finished after 161 steps：10試行の平均step数 = 165.6\n",
      "183 Episode: Finished after 152 steps：10試行の平均step数 = 167.2\n",
      "184 Episode: Finished after 133 steps：10試行の平均step数 = 166.1\n",
      "185 Episode: Finished after 152 steps：10試行の平均step数 = 162.8\n",
      "186 Episode: Finished after 169 steps：10試行の平均step数 = 162.5\n",
      "187 Episode: Finished after 168 steps：10試行の平均step数 = 161.9\n",
      "188 Episode: Finished after 161 steps：10試行の平均step数 = 161.0\n",
      "189 Episode: Finished after 200 steps：10試行の平均step数 = 165.6\n",
      "190 Episode: Finished after 184 steps：10試行の平均step数 = 164.0\n",
      "191 Episode: Finished after 144 steps：10試行の平均step数 = 162.4\n",
      "192 Episode: Finished after 166 steps：10試行の平均step数 = 162.9\n",
      "193 Episode: Finished after 146 steps：10試行の平均step数 = 162.3\n",
      "194 Episode: Finished after 136 steps：10試行の平均step数 = 162.6\n",
      "195 Episode: Finished after 148 steps：10試行の平均step数 = 162.2\n",
      "196 Episode: Finished after 130 steps：10試行の平均step数 = 158.3\n",
      "197 Episode: Finished after 154 steps：10試行の平均step数 = 156.9\n",
      "198 Episode: Finished after 138 steps：10試行の平均step数 = 154.6\n",
      "199 Episode: Finished after 194 steps：10試行の平均step数 = 154.0\n",
      "200 Episode: Finished after 200 steps：10試行の平均step数 = 155.6\n",
      "201 Episode: Finished after 200 steps：10試行の平均step数 = 161.2\n",
      "202 Episode: Finished after 160 steps：10試行の平均step数 = 160.6\n",
      "203 Episode: Finished after 139 steps：10試行の平均step数 = 159.9\n",
      "204 Episode: Finished after 187 steps：10試行の平均step数 = 165.0\n",
      "205 Episode: Finished after 124 steps：10試行の平均step数 = 162.6\n",
      "206 Episode: Finished after 198 steps：10試行の平均step数 = 169.4\n",
      "207 Episode: Finished after 159 steps：10試行の平均step数 = 169.9\n",
      "208 Episode: Finished after 200 steps：10試行の平均step数 = 176.1\n",
      "209 Episode: Finished after 147 steps：10試行の平均step数 = 171.4\n",
      "210 Episode: Finished after 148 steps：10試行の平均step数 = 166.2\n",
      "211 Episode: Finished after 158 steps：10試行の平均step数 = 162.0\n",
      "212 Episode: Finished after 161 steps：10試行の平均step数 = 162.1\n",
      "213 Episode: Finished after 134 steps：10試行の平均step数 = 161.6\n",
      "214 Episode: Finished after 123 steps：10試行の平均step数 = 155.2\n",
      "215 Episode: Finished after 157 steps：10試行の平均step数 = 158.5\n",
      "216 Episode: Finished after 200 steps：10試行の平均step数 = 158.7\n",
      "217 Episode: Finished after 155 steps：10試行の平均step数 = 158.3\n",
      "218 Episode: Finished after 156 steps：10試行の平均step数 = 153.9\n",
      "219 Episode: Finished after 124 steps：10試行の平均step数 = 151.6\n",
      "220 Episode: Finished after 200 steps：10試行の平均step数 = 156.8\n",
      "221 Episode: Finished after 200 steps：10試行の平均step数 = 161.0\n",
      "222 Episode: Finished after 119 steps：10試行の平均step数 = 156.8\n",
      "223 Episode: Finished after 115 steps：10試行の平均step数 = 154.9\n",
      "224 Episode: Finished after 200 steps：10試行の平均step数 = 162.6\n",
      "225 Episode: Finished after 155 steps：10試行の平均step数 = 162.4\n",
      "226 Episode: Finished after 200 steps：10試行の平均step数 = 162.4\n",
      "227 Episode: Finished after 200 steps：10試行の平均step数 = 166.9\n",
      "228 Episode: Finished after 200 steps：10試行の平均step数 = 171.3\n",
      "229 Episode: Finished after 200 steps：10試行の平均step数 = 178.9\n",
      "230 Episode: Finished after 130 steps：10試行の平均step数 = 171.9\n",
      "231 Episode: Finished after 190 steps：10試行の平均step数 = 170.9\n",
      "232 Episode: Finished after 200 steps：10試行の平均step数 = 179.0\n",
      "233 Episode: Finished after 194 steps：10試行の平均step数 = 186.9\n",
      "234 Episode: Finished after 196 steps：10試行の平均step数 = 186.5\n",
      "235 Episode: Finished after 200 steps：10試行の平均step数 = 191.0\n",
      "236 Episode: Finished after 200 steps：10試行の平均step数 = 191.0\n",
      "237 Episode: Finished after 157 steps：10試行の平均step数 = 186.7\n",
      "238 Episode: Finished after 128 steps：10試行の平均step数 = 179.5\n",
      "239 Episode: Finished after 162 steps：10試行の平均step数 = 175.7\n",
      "240 Episode: Finished after 200 steps：10試行の平均step数 = 182.7\n",
      "241 Episode: Finished after 200 steps：10試行の平均step数 = 183.7\n",
      "242 Episode: Finished after 152 steps：10試行の平均step数 = 178.9\n",
      "243 Episode: Finished after 200 steps：10試行の平均step数 = 179.5\n",
      "244 Episode: Finished after 200 steps：10試行の平均step数 = 179.9\n",
      "245 Episode: Finished after 180 steps：10試行の平均step数 = 177.9\n",
      "246 Episode: Finished after 200 steps：10試行の平均step数 = 177.9\n",
      "247 Episode: Finished after 168 steps：10試行の平均step数 = 179.0\n",
      "248 Episode: Finished after 200 steps：10試行の平均step数 = 186.2\n",
      "249 Episode: Finished after 200 steps：10試行の平均step数 = 190.0\n",
      "250 Episode: Finished after 200 steps：10試行の平均step数 = 190.0\n",
      "251 Episode: Finished after 200 steps：10試行の平均step数 = 190.0\n",
      "252 Episode: Finished after 200 steps：10試行の平均step数 = 194.8\n",
      "253 Episode: Finished after 200 steps：10試行の平均step数 = 194.8\n",
      "254 Episode: Finished after 177 steps：10試行の平均step数 = 192.5\n",
      "255 Episode: Finished after 200 steps：10試行の平均step数 = 194.5\n",
      "256 Episode: Finished after 200 steps：10試行の平均step数 = 194.5\n",
      "257 Episode: Finished after 200 steps：10試行の平均step数 = 197.7\n",
      "258 Episode: Finished after 200 steps：10試行の平均step数 = 197.7\n",
      "259 Episode: Finished after 200 steps：10試行の平均step数 = 197.7\n",
      "260 Episode: Finished after 200 steps：10試行の平均step数 = 197.7\n",
      "261 Episode: Finished after 200 steps：10試行の平均step数 = 197.7\n",
      "262 Episode: Finished after 141 steps：10試行の平均step数 = 191.8\n",
      "263 Episode: Finished after 200 steps：10試行の平均step数 = 191.8\n",
      "264 Episode: Finished after 200 steps：10試行の平均step数 = 194.1\n",
      "265 Episode: Finished after 145 steps：10試行の平均step数 = 188.6\n",
      "266 Episode: Finished after 200 steps：10試行の平均step数 = 188.6\n",
      "267 Episode: Finished after 200 steps：10試行の平均step数 = 188.6\n",
      "268 Episode: Finished after 200 steps：10試行の平均step数 = 188.6\n",
      "269 Episode: Finished after 200 steps：10試行の平均step数 = 188.6\n",
      "270 Episode: Finished after 172 steps：10試行の平均step数 = 185.8\n",
      "271 Episode: Finished after 193 steps：10試行の平均step数 = 185.1\n",
      "272 Episode: Finished after 200 steps：10試行の平均step数 = 191.0\n",
      "273 Episode: Finished after 200 steps：10試行の平均step数 = 191.0\n",
      "274 Episode: Finished after 199 steps：10試行の平均step数 = 190.9\n",
      "275 Episode: Finished after 194 steps：10試行の平均step数 = 195.8\n",
      "276 Episode: Finished after 178 steps：10試行の平均step数 = 193.6\n",
      "277 Episode: Finished after 200 steps：10試行の平均step数 = 193.6\n",
      "278 Episode: Finished after 200 steps：10試行の平均step数 = 193.6\n",
      "279 Episode: Finished after 200 steps：10試行の平均step数 = 193.6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "280 Episode: Finished after 200 steps：10試行の平均step数 = 196.4\n",
      "281 Episode: Finished after 200 steps：10試行の平均step数 = 197.1\n",
      "282 Episode: Finished after 200 steps：10試行の平均step数 = 197.1\n",
      "283 Episode: Finished after 200 steps：10試行の平均step数 = 197.1\n",
      "284 Episode: Finished after 200 steps：10試行の平均step数 = 197.2\n",
      "285 Episode: Finished after 200 steps：10試行の平均step数 = 197.8\n",
      "286 Episode: Finished after 200 steps：10試行の平均step数 = 200.0\n",
      "10回連続成功\n",
      "287 Episode: Finished after 200 steps：10試行の平均step数 = 200.0\n"
     ]
    }
   ],
   "source": [
    "# main クラス\n",
    "cartpole_env = Environment()\n",
    "cartpole_env.run()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
